import execjs
import requests
import json
import time
import pymysql

'''
传入URL地址，能过Request post请求获取返回的JSON数据。
url：网站地址
method：方法，post或get
param：传递的参数
'''
def get_data(url, method, param):
    # 定义header 信息
    header = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36",
        "accept": "application/json; charset=utf-8",
        "host": "www.techlabplt.com:8080",
        "Origin": "http://www.techlabplt.com:8080"
    }
    if method == 'post':
        # 发起request post请求
        response = requests.post(url, headers=header, json=param)
    if method == 'get':
        # 发起request post请求
        response = requests.get(url, headers=header)

    # 如果正常返回，遍历输出数据到控制台。
    if response.status_code == 200:
        # 通过返回的数据转换为json格式，或者直接获取response.json。
        jsonData = json.loads(response.text)
        # 返回获取得到的数据
        return jsonData

'''
根据传递过来的网址，获取response数据返回
url：网站地址
'''
def get_response(url):
    # 发送请求
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.5005.63 Safari/537.36"
    }
    response = requests.get(url=url, headers=headers)
    return response

'''
传入JavaScript路径，获取JavaScript文件内容。
jsFile：JavaScript文件路径和名称
'''
def get_js(jsFile):
    with open(jsFile, 'r', encoding='utf-8') as f:
        data = f.read()
        return data


'''
传入insert语句，保存进MySQL数据库
'''
def save_mysql(insertSql):
    # 建立数据库连接
    connect = pymysql.connect(
        # MySQL数据库的IP地址，默认127.0.0.1（代表本地）
        host='127.0.0.1',
        # MySQL数据库的用户名
        user='root',
        # MySQL数据库的密码
        passwd='admin+123@',
        # MySQL数据库服务的端口号；默认为3306
        port=3306,
        # 数据库名称
        db='spider',
        # 字符编码
        charset='utf8'
    )
    # 使用cursor()方法获取操作游标
    cursor = connect.cursor()
    # jobs表内新增一条数据
    try:
        # 执行sql语句
        cursor.execute(insertSql)
        # 提交到数据库执行
        connect.commit()
    except:
        # 发生异常，回滚
        print('发生异常')
        connect.rollback()

    # 关闭数据库连接
    connect.close()


'''
执行主程序
'''
def main(html_url):
    # 1.获取音乐数据
    param = {'jobName': ''}
    jsonData = get_data(html_url, "post", param)
    jobData = jsonData['rows']
    # 2.循环遍历获取职位信息
    for i in range(len(jobData)):
        # 职位信息
        strId = jobData[i]['id']
        strJobName = jobData[i]['jobName']
        strJobArea = jobData[i]['area']
        strCompanyBoon = jobData[i]['companyBoon']
        strCompanyName = jobData[i]['companyName']
        strCompanyType = jobData[i]['companyType']
        strEducation = jobData[i]['education']
        strExp = jobData[i]['exp']
        strSalary = jobData[i]['salary']
        strSkillTags = jobData[i]['skillTags']
        # list转换成str
        strTypeList = "|".join(jobData[i]['typeList'])
        # 3.通过execjs.compile()编译并加载JavaScript文件内容
        execObj = execjs.compile(get_js('zp.js'))
        # 4.使用call调用JavaScript内的minus方法,获取得到hash
        strRid = execObj.call("jobDetailRedirect", strId)
        # 5.获取音乐下载的地址
        param = {}
        url = 'http://121.5.74.22:8080/BD-PC/job/selectById?rid=' + strRid
        jobDesc = get_data(url, 'get', param)['requirement']
        # 6.保存职位数据至MySQL
        insertSql = """INSERT INTO jobs(job_name,
job_area, company_name, company_boon, company_type, education,
                 exp, salary, skill_tags, type_list, job_description)
                 VALUES ('""" + strJobName + """', '"""\
                    + strJobArea + """','""" + strCompanyName + """','""" \
                    + strCompanyBoon + """','""" + strCompanyType + """','""" \
                    + strEducation + """','""" + strExp + """','""" \
                    + strSalary + """','""" + strSkillTags + """','""" \
                    + strTypeList + """','""" + jobDesc + """')"""
        save_mysql(insertSql)
        print("获取得到第"+str(i+1)+"条数据，并保存成功！")
        # 7.等待2秒
        time.sleep(2)

if __name__ == '__main__':
    url="http://www.techlabplt.com:8080/BD-PC/job/selectByPageAndConduction?"
    currentPage="1&pageSize=20"
    main(url)
